{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "conversational-datasets-baselines.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/matthen/052f5204576a347eef8da6581e6504da/conversational-datasets-baselines.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0uJiamL819G",
        "colab_type": "text"
      },
      "source": [
        "## conversational-datasets baselines\n",
        "\n",
        "This is a colab script for reproducing the benchmarks found in [BENCHMARKS.md](https://github.com/PolyAI-LDN/conversational-datasets/blob/master/BENCHMARKS.md) in the [conversational-datasets repo](https://github.com/PolyAI-LDN/conversational-datasets).\n",
        "\n",
        "This runs using a GPU, to accelerate the methods that use neural network encoders.\n",
        "\n",
        "See [baselines/README.md](https://github.com/PolyAI-LDN/conversational-datasets/blob/master/baselines/README.md)\n",
        "for descriptions of each baseline."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pPDM-lKx81jZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "0f36a3fb-0705-472c-af85-89cbf42f68bb"
      },
      "source": [
        "# Clone the conversational datasets repository.\n",
        "!git clone https://github.com/PolyAI-LDN/conversational-datasets.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'conversational-datasets'...\n",
            "remote: Enumerating objects: 89, done.\u001b[K\n",
            "remote: Counting objects: 100% (89/89), done.\u001b[K\n",
            "remote: Compressing objects: 100% (67/67), done.\u001b[K\n",
            "remote: Total 357 (delta 49), reused 34 (delta 22), pack-reused 268\u001b[K\n",
            "Receiving objects: 100% (357/357), 194.54 KiB | 534.00 KiB/s, done.\n",
            "Resolving deltas: 100% (202/202), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ehfOyY7g-roi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d5cabc97-6892-42c2-b8a1-4b37296528eb"
      },
      "source": [
        "import os\n",
        "from getpass import getpass\n",
        "\n",
        "# We use a consistent random sample of the train and test sets for the\n",
        "# baselines.\n",
        "# Please ask matthen@gmail.com for the URL, and enter it here.\n",
        "# (You could also generate your own random sample, and exact results may vary\n",
        "# slightly).\n",
        "os.environ['DATA_URL'] = getpass(\"Enter URL: \")"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter URL: ··········\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "atOkmQ2E_iSf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "5b5b071f-eef5-45da-f2a2-bd54dddcf19a"
      },
      "source": [
        "# Download and extract the data.\n",
        "!mkdir -p conversational-datasets/data\n",
        "!wget -q \"${DATA_URL}\" -O conversational-datasets/data/data.zip\n",
        "!cd conversational-datasets/data && unzip -o data.zip"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  data.zip\n",
            "  inflating: amazon-train-sampled    \n",
            "  inflating: os-train-sampled        \n",
            "  inflating: reddit-train-sampled    \n",
            "  inflating: os-test-sampled         \n",
            "  inflating: amazon-test-sampled     \n",
            "  inflating: reddit-test-sampled     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-UPvYPk99_O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "46092346-2b41-4923-c81c-91d6aa8587ff"
      },
      "source": [
        "# Install extra pip dependencies.\n",
        "!pip install bert-tensorflow glog tf-sentencepiece\n",
        "%env  TFHUB_CACHE_DIR=/content/tfhub\n",
        "%env  PYTHONPATH=/content/conversational-datasets"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: bert-tensorflow in /usr/local/lib/python2.7/dist-packages (1.0.1)\n",
            "Requirement already satisfied: glog in /usr/local/lib/python2.7/dist-packages (0.3.1)\n",
            "Requirement already satisfied: tf-sentencepiece in /usr/local/lib/python2.7/dist-packages (0.1.82.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python2.7/dist-packages (from bert-tensorflow) (1.12.0)\n",
            "Requirement already satisfied: python-gflags>=3.1 in /usr/local/lib/python2.7/dist-packages (from glog) (3.1.2)\n",
            "env: TFHUB_CACHE_DIR=/content/tfhub\n",
            "env: PYTHONPATH=/content/conversational-datasets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6SsuDm-e9PMa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run all the baselines.\n",
        "# It will take over an hour.\n",
        "!cd conversational-datasets && ./baselines/run_baselines.sh\n",
        "!cat conversational-datasets/baselines/results.csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uvSNal6lEuKK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8a28f22b-22ba-402a-fec9-4022c6be2947"
      },
      "source": [
        "# Run just a single baseline:\n",
        "! cd conversational-datasets && python baselines/run_baseline.py \\\n",
        "    --method USE_QA_SIM \\\n",
        "    --train_dataset data/amazon-train-sampled \\\n",
        "    --test_dataset data/amazon-test-sampled"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0717 04:36:15.421634 139979565979520 deprecation_wrapper.py:119] From /usr/local/lib/python2.7/dist-packages/bert/optimization.py:87: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "W0717 04:36:15.442013 139979565979520 deprecation_wrapper.py:119] From /usr/local/lib/python2.7/dist-packages/tf_sentencepiece/sentencepiece_processor_ops.py:259: The name tf.NotDifferentiable is deprecated. Please use tf.no_gradient instead.\n",
            "\n",
            "W0717 04:36:15.444288 139979565979520 deprecation_wrapper.py:119] From /content/conversational-datasets/baselines/vector_based.py:64: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "2019-07-17 04:36:15.445798: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcuda.so.1\n",
            "2019-07-17 04:36:15.464786: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.465245: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties: \n",
            "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
            "pciBusID: 0000:00:04.0\n",
            "2019-07-17 04:36:15.465560: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-07-17 04:36:15.466847: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcublas.so.10.0\n",
            "2019-07-17 04:36:15.468351: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcufft.so.10.0\n",
            "2019-07-17 04:36:15.468738: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcurand.so.10.0\n",
            "2019-07-17 04:36:15.470466: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2019-07-17 04:36:15.471840: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2019-07-17 04:36:15.476561: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcudnn.so.7\n",
            "2019-07-17 04:36:15.476693: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.477226: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.477690: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0\n",
            "2019-07-17 04:36:15.483793: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2300000000 Hz\n",
            "2019-07-17 04:36:15.484072: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55fdf2c6b640 executing computations on platform Host. Devices:\n",
            "2019-07-17 04:36:15.484111: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): <undefined>, <undefined>\n",
            "2019-07-17 04:36:15.558251: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.558797: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55fdf2c6b800 executing computations on platform CUDA. Devices:\n",
            "2019-07-17 04:36:15.558840: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
            "2019-07-17 04:36:15.559072: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.559475: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties: \n",
            "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
            "pciBusID: 0000:00:04.0\n",
            "2019-07-17 04:36:15.559545: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-07-17 04:36:15.559582: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcublas.so.10.0\n",
            "2019-07-17 04:36:15.559626: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcufft.so.10.0\n",
            "2019-07-17 04:36:15.559663: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcurand.so.10.0\n",
            "2019-07-17 04:36:15.559703: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcusolver.so.10.0\n",
            "2019-07-17 04:36:15.559735: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcusparse.so.10.0\n",
            "2019-07-17 04:36:15.559769: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcudnn.so.7\n",
            "2019-07-17 04:36:15.559883: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.560331: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.560718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0\n",
            "2019-07-17 04:36:15.560784: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcudart.so.10.0\n",
            "2019-07-17 04:36:15.561919: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1181] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
            "2019-07-17 04:36:15.561952: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1187]      0 \n",
            "2019-07-17 04:36:15.561971: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1200] 0:   N \n",
            "2019-07-17 04:36:15.562281: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.562743: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1005] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
            "2019-07-17 04:36:15.563116: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:40] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
            "2019-07-17 04:36:15.563169: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1326] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10802 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
            "I0717 04:36:15.564023 139979565979520 vector_based.py:66] Loading https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/1 model from tensorflow hub\n",
            "I0717 04:36:15.564831 139979565979520 resolver.py:79] Using /content/tfhub to cache modules.\n",
            "W0717 04:36:31.697329 139979565979520 deprecation_wrapper.py:119] From /content/conversational-datasets/baselines/vector_based.py:68: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "I0717 04:36:37.204843 139979565979520 saver.py:1499] Saver not created because there are no variables in the graph to restore\n",
            "I0717 04:36:44.272711 139979565979520 saver.py:1499] Saver not created because there are no variables in the graph to restore\n",
            "W0717 04:36:45.990983 139979565979520 deprecation_wrapper.py:119] From /content/conversational-datasets/baselines/vector_based.py:87: The name tf.global_variables_initializer is deprecated. Please use tf.compat.v1.global_variables_initializer instead.\n",
            "\n",
            "W0717 04:36:46.029885 139979565979520 deprecation_wrapper.py:119] From /content/conversational-datasets/baselines/vector_based.py:87: The name tf.tables_initializer is deprecated. Please use tf.compat.v1.tables_initializer instead.\n",
            "\n",
            "I0717 04:36:46.030472 139979565979520 vector_based.py:88] Initializing graph.\n",
            "I0717 04:36:54.760605 139979565979520 run_baseline.py:220] Loading training data\n",
            "  0% 0/10000 [00:00<?, ?it/s]W0717 04:36:54.761677 139979565979520 deprecation_wrapper.py:119] From baselines/run_baseline.py:182: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
            "\n",
            "I0717 04:36:54.762882 139979565979520 run_baseline.py:188] Reading data/amazon-train-sampled\n",
            "W0717 04:36:54.763079 139979565979520 deprecation.py:323] From baselines/run_baseline.py:189: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use eager execution and: \n",
            "`tf.data.TFRecordDataset(path)`\n",
            "100% 10000/10000 [00:00<00:00, 31310.38it/s]\n",
            "I0717 04:36:55.081111 139979565979520 run_baseline.py:208] Read 10000 examples\n",
            "I0717 04:36:55.081820 139979565979520 run_baseline.py:224] Training USE_QA_SIM method\n",
            "I0717 04:36:55.082257 139979565979520 run_baseline.py:227] Loading test data\n",
            "  0% 0/50000 [00:00<?, ?it/s]I0717 04:36:55.083873 139979565979520 run_baseline.py:188] Reading data/amazon-test-sampled\n",
            "100% 50000/50000 [00:01<00:00, 34878.28it/s]\n",
            "I0717 04:36:56.516540 139979565979520 run_baseline.py:208] Read 50000 examples\n",
            "I0717 04:36:56.516777 139979565979520 run_baseline.py:232] Running evaluation\n",
            "  0% 0/500 [00:00<?, ?it/s]2019-07-17 04:36:58.950335: W tensorflow/compiler/jit/mark_for_compilation_pass.cc:1412] (One-time warning): Not using XLA:CPU for cluster because envvar TF_XLA_FLAGS=--tf_xla_cpu_global_jit was not set.  If you want XLA:CPU, either set that envvar, or use experimental_jit_scope to enable XLA:CPU.  To confirm that XLA is active, pass --vmodule=xla_compilation_cache=1 (as a proper command-line flag, not via TF_XLA_FLAGS) or set the envvar XLA_FLAGS=--xla_hlo_profile.\n",
            "2019-07-17 04:36:59.038882: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library libcublas.so.10.0\n",
            "100% 500/500 [02:12<00:00,  3.99it/s]\n",
            "I0717 04:39:08.790338 139979565979520 run_baseline.py:237] Final computed 1-of-100 accuracy is 66.8%\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}
